package com.atguigu1.core.operator

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

/**
 *
 * @description: mapPartitions优化操作
 * @time: 2021-03-12 18:13
 * @author: baojinlong 
 **/
object Spark04MapPartitions {


  def main(args: Array[String]): Unit = {
    val conf: SparkConf = new SparkConf().setMaster("local[*]").setAppName("rdd")
    val sparkContext = new SparkContext(conf)
    val rdd: RDD[Int] = sparkContext.makeRDD(List(1, 2, 3, 4), 2)
    // mapPartitions:可以以分区为单位进行数据转换操作,但是会将整个分区的数据加载到内存中进行引用,如果处理完的数据是不存在释放掉,有可能会造成数据oom
    val rddResult: RDD[Int] = rdd.mapPartitions(item => {
      println("每个分区执行一次")
      item.map(_ * 3)
    })
    rddResult.collect().foreach(println)

    sparkContext.stop()
  }

}
